# -*- coding: utf-8 -*-
import scrapy
import json
from dingxiangyuan import utils
from dingxiangyuan.DBHelper import redis_conn
from dingxiangyuan.items import DingxiangyuanItemLoader, UserTopicPostItem


class UserTopicPostsSpider(scrapy.Spider):
    name = 'user_topic_posts'
    allowed_domains = ['i.dxy.cn']
    # start_urls = ['http://i.dxy.cn/bbs/bbsapi/forsns?JUTE_TOKEN=a543cf85-ab31-4f31-8da4-eb98b36d9f76&s=user_reply_json&t=list&uid=567043&page=1&size=25']

    custom_settings = {
        'DEFAULT_REQUEST_HEADERS': {
            'Cookie': 'DXY_USER_GROUP=42; __auc=d205b571172799412796bf70504; _ga=GA1.2.642389535.1591177647; CLASS_CASTGC=3a167ea5a9e2314851400092a956e733e0d80ef565ff659b3984aa499ed34af7535bc7d25e1766ba16799e742c8ed0dbd24f6a8f72a32bcbd74d9d0791f1a5ff45f32e599b521d41047fa3041ec868db4fc021fe0e2d3c7f579a8c8c85de0419ef4292dff1c067138053129c49e48b99d7b7408903e6b8b7f2e7960ae22b38aadde6db6bc900526524f0897e206d6403e041e3d59d3cd08ae376c34e363d86a4999793999ba69ffe2f1bce9f855352e0f21e5c89838b9169721f0ab3d14f42a4963a507e306d5dd3d93be83908b761ff877ce0ba9d05e8bf5a7e4ce6bc169be1fff492edb712e9629ece8af98184f972283f20446e57d9f7804c32d2194f9024; JUTE_BBS_DATA=8867be01e7314365695377e3399cf32853329d05a7b58dab9fad869d790feed899be1912f006c9c4cefe3222d89b35aa8f58e0bcbd9b849b49c6e69ea1950cb580671d366be2287a014e1c965d02f595; home_hash=%23money; route_bbs=4b7df70fe96acfe89c32b68456fd2ed3; JUTE_SESSION_ID=760526e7-b824-4577-9a83-96190c609e6f; JUTE_TOKEN=a543cf85-ab31-4f31-8da4-eb98b36d9f76; route=6db93de1de9a21f214bd53b6a2c354d3; JSESSIONID=73DB57252CD9A853541EABFC7C984A3B; __utma=17875052.642389535.1591177647.1591675467.1591675467.1; __utmc=17875052; __utmz=17875052.1591675467.1.1.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; __utmt=1; __utmb=17875052.1.10.1591675467; Hm_lvt_8a6dad3652ee53a288a11ca184581908=1591610496,1591615121,1591663078,1591675467; Hm_lpvt_8a6dad3652ee53a288a11ca184581908=1591675467; CMSSESSIONID=0874B9953D34D913C2E1D5F5232887CD-n1; __asc=1babefa517297403e6eeda32ab6; Hm_lvt_253e434fd63b62a2659ddd3e7412f769=1591615126,1591622542,1591663082,1591675470; __utma=251724881.642389535.1591177647.1591666045.1591675470.16; __utmc=251724881; __utmz=251724881.1591675470.16.6.utmcsr=dxy.cn|utmccn=(referral)|utmcmd=referral|utmcct=/; Hm_lpvt_253e434fd63b62a2659ddd3e7412f769=1591675493; __utmb=251724881.4.10.1591675470; JUTE_SESSION=c94a35c74c51c0203aed2168471de2dfee4dcb429f74c150e7a514a260d961119b808e875dd3eed51c6bc02200f367e2d448a85169df3ca180453d6883a2442803abd5a51ce8cb03',
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36',
        }
    }

    def start_requests(self):
        JUTE_TOKEN = '9c271aa3-ffbf-4ea3-b021-13b57697be51'
        page_size = 15
        user_infos = redis_conn.smembers('user_top_user_start_infos')
        for user in user_infos:
            user_id, user_name, topic_pages, post_pages = user.decode().split(';')

            for topic_page in range(1, int(topic_pages) + 1):
                user_topic_url = f'http://i.dxy.cn/bbs/bbsapi/forsns?JUTE_TOKEN={JUTE_TOKEN}&s=user_topic&t=list&uid={user_id}&page={topic_page}&size={page_size}'
                yield scrapy.Request(url=user_topic_url, callback=self.parse)

            for post_page in range(1, int(post_pages) + 1):
                user_post_url = f'http://i.dxy.cn/bbs/bbsapi/forsns?JUTE_TOKEN={JUTE_TOKEN}&s=user_reply_json&t=list&uid={user_id}&page={post_page}&size={page_size}'
                yield scrapy.Request(url=user_post_url, callback=self.parse_post, meta={'user_id': user_id, 'user_name':user_name})

    def parse(self, response):
        data = json.loads(response.text)
        for item in data['items']:
            item_loader = DingxiangyuanItemLoader(item=UserTopicPostItem())
            item_loader.add_value('user_id', item['userId'])
            item_loader.add_value('user_name', item['username'])
            item_loader.add_value('topic_user_id', item['userId'])
            item_loader.add_value('topic_user_name', item['username'])
            item_loader.add_value('subject', item['subject'])
            item_loader.add_value('reply', item['reply'])
            item_loader.add_value('topic_id', item['id'])
            item_loader.add_value('board', item['boardTitle'])
            item_loader.add_value('last_post_time', utils.timestamp_to_datetime(int(str(item['lastPostDate'])[:10])))
            item_loader.add_value('post_type', 1)
            user_post_item = item_loader.load_item()
            yield user_post_item

    def parse_post(self, response):
        user_id = response.meta['user_id']
        user_name = response.meta['user_name']
        data = json.loads(response.text)
        for item in data['posts']:
            item_loader = DingxiangyuanItemLoader(item=UserTopicPostItem())
            item_loader.add_value('user_id', user_id)
            item_loader.add_value('user_name', user_name),
            item_loader.add_value('topic_user_id', item['userId'])
            item_loader.add_value('topic_user_name', item['username'])
            item_loader.add_value('subject', item['subject'])
            item_loader.add_value('reply', item['reply'])
            item_loader.add_value('topic_id', item['id'])
            item_loader.add_value('board', item['boardTitle'])
            item_loader.add_value('last_post_time', utils.timestamp_to_datetime(int(str(item['lastPostDate'])[:10])))
            item_loader.add_value('post_type', 2)
            user_post_item = item_loader.load_item()
            yield user_post_item